{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "nfzqnU3AbkeL",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 233.0
    },
    "outputId": "f003ad6e-a38d-42c1-be4b-08f3002d5405"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1000 cost = 1.657833\n",
      "Epoch: 2000 cost = 1.411641\n",
      "Epoch: 3000 cost = 1.115576\n",
      "Epoch: 4000 cost = 0.927828\n",
      "Epoch: 5000 cost = 0.696575\n",
      "Epoch: 6000 cost = 0.410544\n",
      "Epoch: 7000 cost = 0.316320\n",
      "Epoch: 8000 cost = 0.196335\n",
      "Epoch: 9000 cost = 0.147900\n",
      "Epoch: 10000 cost = 0.119816\n",
      "Lorem ipsum dolor sit amet consectetur adipisicing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Ut enim ad minim veniam quis nostrud exercitation\n",
      "['ipsum', 'dolor', 'sit', 'amet', 'consectetur', 'adipisicing', 'elit', 'sed', 'eiusmod', 'eiusmod', 'tempor', 'incididunt', 'ut', 'labore', 'et', 'dolore', 'magna', 'aliqua', 'Ut', 'enim', 'ad', 'minim', 'veniam', 'quis', 'nostrud', 'exercitation']\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "  code by Tae Hwan Jung(Jeff Jung) @graykode\n",
    "'''\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "sentence = (\n",
    "    'Lorem ipsum dolor sit amet consectetur adipisicing elit '\n",
    "    'sed do eiusmod tempor incididunt ut labore et dolore magna '\n",
    "    'aliqua Ut enim ad minim veniam quis nostrud exercitation'\n",
    ")\n",
    "\n",
    "word_dict = {w: i for i, w in enumerate(list(set(sentence.split())))}\n",
    "number_dict = {i: w for i, w in enumerate(list(set(sentence.split())))}\n",
    "n_class = len(word_dict)\n",
    "n_step = len(sentence.split())\n",
    "n_hidden = 5\n",
    "\n",
    "def make_batch(sentence):\n",
    "    input_batch = []\n",
    "    target_batch = []\n",
    "\n",
    "    words = sentence.split()\n",
    "    for i, word in enumerate(words[:-1]):\n",
    "        input = [word_dict[n] for n in words[:(i + 1)]]\n",
    "        input = input + [0] * (n_step - len(input))\n",
    "        target = word_dict[words[i + 1]]\n",
    "        input_batch.append(np.eye(n_class)[input])\n",
    "        target_batch.append(np.eye(n_class)[target])\n",
    "\n",
    "    return input_batch, target_batch\n",
    "\n",
    "# Bi-LSTM Model\n",
    "X = tf.placeholder(tf.float32, [None, n_step, n_class])\n",
    "Y = tf.placeholder(tf.float32, [None, n_class])\n",
    "\n",
    "W = tf.Variable(tf.random_normal([n_hidden * 2, n_class]))\n",
    "b = tf.Variable(tf.random_normal([n_class]))\n",
    "\n",
    "lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(n_hidden)\n",
    "lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(n_hidden)\n",
    "\n",
    "# outputs : [batch_size, len_seq, n_hidden], states : [batch_size, n_hidden]\n",
    "outputs, _ = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,lstm_bw_cell, X, dtype=tf.float32)\n",
    "\n",
    "outputs = tf.concat([outputs[0], outputs[1]], 2) # output[0] : lstm_fw, output[1] : lstm_bw\n",
    "outputs = tf.transpose(outputs, [1, 0, 2]) # [n_step, batch_size, n_hidden]\n",
    "outputs = outputs[-1] # [batch_size, n_hidden]\n",
    "\n",
    "model = tf.matmul(outputs, W) + b\n",
    "\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=model, labels=Y))\n",
    "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n",
    "\n",
    "prediction = tf.cast(tf.argmax(model, 1), tf.int32)\n",
    "\n",
    "# Training\n",
    "init = tf.global_variables_initializer()\n",
    "sess = tf.Session()\n",
    "sess.run(init)\n",
    "\n",
    "input_batch, target_batch = make_batch(sentence)\n",
    "\n",
    "for epoch in range(10000):\n",
    "    _, loss = sess.run([optimizer, cost], feed_dict={X: input_batch, Y: target_batch})\n",
    "    if (epoch + 1)%1000 == 0:\n",
    "        print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
    "\n",
    "predict =  sess.run([prediction], feed_dict={X: input_batch})\n",
    "print(sentence)\n",
    "print([number_dict[n] for n in [pre for pre in predict[0]]])"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Bi-LSTM-Tensor.ipynb",
   "version": "0.3.2",
   "provenance": [],
   "collapsed_sections": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "GPU"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
